import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import keras
from keras import Sequential
import re
from keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Input
from tensorflow.keras.optimizers import Adam

#将数据切割为hobbies, foods, and household,并再分为train, validation and test，已经不需要切割了，即不用运行了
def split_data(category):  
    df                = pd.read_csv('train_with_price.csv')
    category_data     = df[df['cat_id'] == category]
    train_df, temp_df = train_test_split(category_data, test_size=0.2, random_state=0)
    test_df, val_df   = train_test_split(temp_df, test_size=0.5, random_state=0)
    train_df.to_csv(f'{category}_train_dataset.csv', index=False)
    test_df.to_csv (f'{category}_test_dataset.csv', index=False)
    val_df.to_csv  (f'{category}_validation_dataset.csv', index=False)

def split_all_data():
    category=['HOUSEHOLD','HOBBIES','FOODS']
    for x in category:
        split_data(x)
split_all_data()

C:\Users\a8090\AppData\Local\Temp\ipykernel_17932\1529081861.py:3: DtypeWarning: Columns (15,16) have mixed types. Specify dtype option on import or set low_memory=False.
  df                = pd.read_csv('train_with_price.csv')
C:\Users\a8090\AppData\Local\Temp\ipykernel_17932\1529081861.py:3: DtypeWarning: Columns (15,16) have mixed types. Specify dtype option on import or set low_memory=False.
  df                = pd.read_csv('train_with_price.csv')
C:\Users\a8090\AppData\Local\Temp\ipykernel_17932\1529081861.py:3: DtypeWarning: Columns (15,16) have mixed types. Specify dtype option on import or set low_memory=False.
  df                = pd.read_csv('train_with_price.csv')

#这里要手动把household改成foods和hobbies
train_data      = pd.read_csv("HOUSEHOLD_train_dataset.csv")
validation_data = pd.read_csv("HOUSEHOLD_validation_dataset.csv")
testing_data    = pd.read_csv("HOUSEHOLD_test_dataset.csv")

#预处理数据-去除不需要的栏，并将数据全部变为数字----
def preprocess_data(data):
    data['item_id']   = data['item_id'].astype(str).apply(lambda x: ''.join(re.findall(r'\d+', x)))
    data['dept_id']   = data['dept_id'].astype(str).apply(lambda x: ''.join(re.findall(r'\d+', x)))
    data.drop(['cat_id'], axis=1, inplace=True)
    data['store_id']  = data['store_id'].astype(str).apply(lambda x: ''.join(re.findall(r'\d+', x)))
    data['state_id']  = data['state_id'].replace({'CA': 1, 'WI': 2, 'TX': 3})
    data['d']         = data['d'].astype(str).apply(lambda x: ''.join(re.findall(r'\d+', x)))
    data.drop(['date'], axis=1, inplace=True)
    data['weekday']   = data['weekday'].replace({'Monday': 1, 'Tuesday': 2, 'Wednesday': 3, 'Thursday': 4, 'Friday': 5 , 'Saturday': 6, 'Sunday': 7})
    data.drop(['event_name_1'], axis=1, inplace=True)
    data.drop(['event_name_2'], axis=1, inplace=True)
    data['event_type_1'] = data['event_type_1'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
    data['event_type_2'] = data['event_type_2'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
    data['event_type_1'] = data['event_type_1'].fillna(0)   #将活动中空的数据变为0
    data['event_type_2'] = data['event_type_2'].fillna(0)
    data['event_type']   = data['event_type_1']+data['event_type_2']
    data.drop(['event_type_1'], axis=1, inplace=True)
    data.drop(['event_type_2'], axis=1, inplace=True)
    data = data.dropna()
    
preprocess_data(train_data)
preprocess_data(validation_data)
preprocess_data(testing_data)

C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:12: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['state_id']  = data['state_id'].replace({'CA': 1, 'WI': 2, 'TX': 3})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:15: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['weekday']   = data['weekday'].replace({'Monday': 1, 'Tuesday': 2, 'Wednesday': 3, 'Thursday': 4, 'Friday': 5 , 'Saturday': 6, 'Sunday': 7})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:18: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_1'] = data['event_type_1'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:19: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_2'] = data['event_type_2'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:12: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['state_id']  = data['state_id'].replace({'CA': 1, 'WI': 2, 'TX': 3})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:15: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['weekday']   = data['weekday'].replace({'Monday': 1, 'Tuesday': 2, 'Wednesday': 3, 'Thursday': 4, 'Friday': 5 , 'Saturday': 6, 'Sunday': 7})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:18: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_1'] = data['event_type_1'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:19: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_2'] = data['event_type_2'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:12: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['state_id']  = data['state_id'].replace({'CA': 1, 'WI': 2, 'TX': 3})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:15: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['weekday']   = data['weekday'].replace({'Monday': 1, 'Tuesday': 2, 'Wednesday': 3, 'Thursday': 4, 'Friday': 5 , 'Saturday': 6, 'Sunday': 7})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:18: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_1'] = data['event_type_1'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})
C:\Users\a8090\AppData\Local\Temp\ipykernel_10024\1590625379.py:19: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`
  data['event_type_2'] = data['event_type_2'].replace({'Religious': 1, 'Cultural': 1, 'National': 1, 'Sporting': 1})

train_data.head()

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Input, Dropout
from tensorflow.keras.optimizers import Adam
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
import matplotlib.pyplot as plt
from tensorflow.keras.callbacks import EarlyStopping

def create_xy_data(df, pre_type = ""):
    x_train_id = (df['id'] + "_" + pre_type).values
    idx = np.unique(x_train_id, return_index=True)[1]
    idx.sort()
    x_train_id = x_train_id[idx]
    y_train = df['num_sold'].values
    df = df.drop(['id','num_sold','item_id', 'dept_id','wm_yr_wk', 'year'],axis=1)
    return df, y_train, x_train_id

def create_x(df, pre_type=""):
    x_train_id = (df['id'] + "_" + pre_type).values
    idx = np.unique(x_train_id, return_index=True)[1]
    idx.sort()
    x_train_id = x_train_id[idx]
    df = df.drop(['id','num_sold','item_id', 'dept_id', 'wm_yr_wk','year'],axis=1)
    return df, x_train_id

def create_time_steps(length):
    time_steps = []
    for i in range(-length, 0, 1):
        time_steps.append(i)
    return time_steps

def show_plot(plot_data, delta, title):
    labels = ['History', 'True Future', 'Model Prediction']
    marker = ['.-', 'rx', 'go']
    time_steps = create_time_steps(plot_data[0].shape[0])

    if delta:
        future = delta
    else:
        future = 0

    plt.title(title)
    for i, x in enumerate(plot_data):
        if i:
            plt.plot(future, plot_data[i], marker[i], markersize=10, label=labels[i])
        else:
            plt.plot(time_steps, plot_data[i].flatten(), marker[i], label=labels[i])
    plt.legend()
    plt.xlim([time_steps[0] - 1, (future + 2) * 2])
    plt.xlabel('Time-Step')
    return plt

LOOKBACK_MAX     = 28  #28? 14?
LOOKBACK_ARR     = np.array([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14])

y_val_pre_all     = []
y_eval_pre_all    = []
y_val_true_all    = []
val_id_all        = []
eval_id_all       = []
SE_list           = [] 
number_of_trained = 0
y_true_all    = []
y_pred_all    = []
unique_items  = train_data['item_id'].unique()
target_scaler = MinMaxScaler(feature_range=(0, 1))

y_eval_output = []

# create a model for each product
while number_of_trained < 100 and number_of_trained < len(unique_items):
    item = unique_items[number_of_trained]
    print("-----------------------------------")
    print("Current item is ", item)

    # Filter data for the current item
    item_train_data = train_data[train_data['item_id'] == item].reset_index(drop=True)
    item_val_data     = validation_data[validation_data['item_id'] == item].reset_index(drop=True)
    item_eval_data    = testing_data[testing_data['item_id'] == item].reset_index(drop=True)

    item_train_data['state_id'] = item_train_data['state_id'].astype(str)
    item_train_data['store_id'] = item_train_data['store_id'].astype(str)

    item_val_data['state_id'] = item_val_data['state_id'].astype(str)
    item_val_data['store_id'] = item_val_data['store_id'].astype(str)

    item_eval_data['state_id'] = item_eval_data['state_id'].astype(str)
    item_eval_data['store_id'] = item_eval_data['store_id'].astype(str)

    # Since it is not reasonable to use data from different stores when extracting the previous 14 days
    state_store_list         = (item_train_data['state_id'] + '_' + item_train_data['store_id']).to_numpy()
    val_state_store_list = (item_val_data['state_id'] + '_' + item_val_data['store_id']).to_numpy()
    eval_state_store_list = (item_eval_data['state_id'] + '_' + item_eval_data['store_id']).to_numpy()

    # Transform categorical data using One-hot-encoding
    train_dummy         = pd.get_dummies(item_train_data, columns=['store_id','state_id', 'weekday', 'snap'], drop_first=False)
    train_columns         = train_dummy.columns
    val_dummy           = pd.get_dummies(item_val_data,     columns=['store_id','state_id', 'weekday', 'snap'], drop_first=False)
    eval_dummy          = pd.get_dummies(item_eval_data,    columns=['store_id','state_id', 'weekday', 'snap'], drop_first=False)
    print("Data types of train_dummy after one-hot encoding:")
    print(train_dummy.dtypes)

    missing_cols = set(train_dummy.columns) - set(eval_dummy.columns)
    for col in missing_cols:
        eval_dummy[col] = 0  # 补全缺失的列，并设为0
    eval_dummy = eval_dummy[train_dummy.columns]  # 保持列顺序一致
    eval_dummy = eval_dummy.reindex(columns=train_dummy.columns, fill_value=0)
    # Create dataframe
    x_data, y_data, data_id = create_xy_data(train_dummy)
    x_val, y_val, val_id     = create_xy_data(val_dummy, pre_type="validation")
    x_eval, eval_id          = create_x(eval_dummy, pre_type="evaluation")

    val_id_all.append(val_id)
    eval_id_all.append(eval_id)

    # Standardize price
    scaler_x = StandardScaler()
    x_data['sell_price']     = scaler_x.fit_transform(np.array(x_data['sell_price']).reshape(-1,1))
    x_val['sell_price']       = scaler_x.transform(np.array(x_val['sell_price']).reshape(-1,1))
    x_eval['sell_price']    = scaler_x.transform(np.array(x_eval['sell_price']).reshape(-1,1))
    x_columns = x_data.columns
    print(x_columns)

    # Scale target variable
    print(f"Original y_data range: min={np.min(y_data)}, max={np.max(y_data)}")
    y_data_scaled = target_scaler.fit_transform(y_data.reshape(-1, 1)).flatten()
    print(f"Scaled y_data range: min={np.min(y_data_scaled)}, max={np.max(y_data_scaled)}")
    y_val_scaled = target_scaler.transform(y_val.reshape(-1, 1)).flatten()
    print(f"Scaled y_val range: min={np.min(y_val_scaled)}, max={np.max(y_val_scaled)}")

    x_time = []
    y_time = []
    x_val_time = []
    y_val_time = []
    x_eval_time = []
    i = LOOKBACK_MAX

    # Store-specific data extraction and handling
    unique_stores = np.unique(state_store_list)
    for store in unique_stores:
        train_indices = np.where(state_store_list == store)[0]
        val_indices   = np.where(val_state_store_list == store)[0]
        eval_indices  = np.where(eval_state_store_list == store)[0]

        store_x_data = x_data.iloc[train_indices].reset_index(drop=True)
        store_y_data = y_data_scaled[train_indices]
        store_x_val  = x_val.iloc[val_indices].reset_index(drop=True)
        store_y_val  = y_val_scaled[val_indices]
        store_x_eval = x_eval.iloc[eval_indices].reset_index(drop=True)

        # Convert boolean columns to integers
        bool_cols = ['store_id_1', 'store_id_2', 'store_id_3', 'store_id_4',
                     'state_id_1', 'state_id_2', 'state_id_3',
                     'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7',
                     'snap_0', 'snap_1']
        for col in bool_cols:
            if col in store_x_data.columns:
                store_x_data[col] = store_x_data[col].astype(int)
            if col in store_x_val.columns:
                store_x_val[col] = store_x_val[col].astype(int)
            if col in store_x_eval.columns:
                store_x_eval[col] = store_x_eval[col].astype(int)

        #看看有没有问题
        for col in ['event_type_1', 'event_type_2']:
            if col in store_x_data.columns:
                store_x_data[col] = pd.to_numeric(store_x_data[col], errors='coerce')
            if col in store_x_val.columns:
                store_x_val[col] = pd.to_numeric(store_x_val[col], errors='coerce')
            if col in store_x_eval.columns:
                store_x_eval[col] = pd.to_numeric(store_x_eval[col], errors='coerce')

        # Process training data for lookback
        for i in range(LOOKBACK_MAX, len(store_x_data)):
            lookback_data = []
            for lb in LOOKBACK_ARR:
                if i - lb >= 0:
                    lookback_data.append(store_x_data.iloc[i - lb].values.tolist())
                else:
                    lookback_data.append([0] * store_x_data.shape[1])
            x_time.append(lookback_data)
            y_time.append(store_y_data[i])

        # Process validation data for lookback
        for i in range(LOOKBACK_MAX, len(store_x_val)):
            lookback_data = []
            for lb in LOOKBACK_ARR:
                if i - lb >= 0:
                    lookback_data.append(store_x_val.iloc[i - lb].values.tolist())
                else:
                    lookback_data.append([0] * store_x_val.shape[1])
            x_val_time.append(lookback_data)
            y_val_time.append(store_y_val[i])

        for i in range(LOOKBACK_MAX, len(store_x_eval)):
            lookback_data = []
            for lb in LOOKBACK_ARR:
                if i - lb >= 0:
                    lookback_data.append(store_x_eval.iloc[i - lb].values.tolist())
                else:
                    lookback_data.append([0] * store_x_eval.shape[1])
            x_eval_time.append(lookback_data)

    x_time     = np.array(x_time).astype(np.float32)
    y_time     = np.array(y_time).astype(np.float32)
    x_val_time = np.array(x_val_time).astype(np.float32)
    y_val_time = np.array(y_val_time).astype(np.float32)
    x_eval_time= np.array(x_eval_time).astype(np.float32)


    print(f"x_eval_time shape before reshape: {x_eval_time.shape}")
    print(f"x_data.shape[1]: {x_data.shape[1]}")
    print(f"LOOKBACK_ARR.shape[0]: {LOOKBACK_ARR.shape[0]}")
    print(f"item_eval_data.shape: {item_eval_data.shape}")
    unique_stores = np.unique(eval_state_store_list)
    for store in unique_stores:
        eval_indices = np.where(eval_state_store_list == store)[0]
        store_x_eval = x_eval.iloc[eval_indices].reset_index(drop=True)
        print(f"store_x_eval.shape: {store_x_eval.shape}")


    x_time = x_time.reshape((x_time.shape[0], LOOKBACK_ARR.shape[0], x_data.shape[1]))
    x_val_time = x_val_time.reshape((x_val_time.shape[0], LOOKBACK_ARR.shape[0], x_data.shape[1]))
    x_eval_time = x_eval_time.reshape((x_eval_time.shape[0], LOOKBACK_ARR.shape[0], x_eval.shape[1]))

    model = Sequential()
    model.add(Input(shape=(LOOKBACK_ARR.shape[0], x_data.shape[1])))
    model.add(LSTM(256, activation='relu', return_sequences=True))
    model.add(Dropout(0.2))
    model.add(LSTM(256, activation='relu'))
    model.add(Dropout(0.2))
    model.add(Dense(1))
    model.compile(optimizer=Adam(learning_rate=0.001), loss='mse')
    model.summary()

    number_of_trained +=1

    # Early stopping
    early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)

    history = model.fit(x = x_time,
                        y = y_time,
                        epochs=100,
                        shuffle=True,
                        batch_size=128,
                        validation_split = 0.1,
                        verbose=1,
                        callbacks=[early_stopping])
    # Predict
    y_val_pre_scaled = model.predict(x_val_time).reshape(-1,)
    y_val_pre = target_scaler.inverse_transform(y_val_pre_scaled.reshape(-1, 1)).flatten()
    y_val_true_original = target_scaler.inverse_transform(y_val_time.reshape(-1, 1)).flatten()
    y_val_pre = np.maximum(0, y_val_pre)

    y_val_pre_all.extend(y_val_pre)
    y_val_true_all.extend(y_val_true_original)
    print("Sample raw predictions (after inverse transform and clipping):", y_val_pre[:5])
    MSE = mean_squared_error(y_val_true_original, y_val_pre)
    SE_list.append(MSE*len(y_val_pre))
    print("RMSE = ", np.sqrt(MSE))

    r2 = r2_score(y_val_true_original, y_val_pre)
    print(f"Validation R-squared for item {item}: {r2}")

    y_eval_pre_scaled = model.predict(x_eval_time).reshape(-1,)
    y_eval_pre = target_scaler.inverse_transform(y_eval_pre_scaled.reshape(-1, 1)).flatten()
    y_eval_pre = np.maximum(0, y_eval_pre)
    if len(y_eval_pre) >= 28:
        y_eval_output.append(y_eval_pre[-28:])
    else:
        padding = np.zeros(28 - len(y_eval_pre))
        y_eval_output.append(np.concatenate([y_eval_pre, padding]))

    print(f"Predicted y_val range (after inverse transform and clipping): min={np.min(y_val_pre)}, max={np.max(y_val_pre)}")
    print(f"True y_val range (after inverse transform): min={np.min(y_val_true_original)}, max={np.max(y_val_true_original)}")

    #可视化图表-折线图
    plot_idx = np.random.randint(0, len(x_val_time))
    plot_sample = x_val_time[plot_idx]
    true_value_scaled = y_val_time[plot_idx]
    prediction_scaled = y_val_pre_scaled[plot_idx]

    true_value_original = target_scaler.inverse_transform(np.array([true_value_scaled]).reshape(-1, 1)).flatten()[0]
    prediction_original = target_scaler.inverse_transform(np.array([prediction_scaled]).reshape(-1, 1)).flatten()[0]

    plot_data = [
        plot_sample[:, -1],
        np.array([true_value_original]),
        np.array([prediction_original])]
    plt = show_plot(plot_data, delta=1, title=f'Item {item} Prediction Example')
    plt.show()

from sklearn.metrics import r2_score, mean_absolute_error
#给出总体的结果
overall_mse = mean_squared_error(y_val_true_all, y_val_pre_all)
overall_rmse = np.sqrt(overall_mse)
overall_r2 = r2_score(y_val_true_all, y_val_pre_all)
overall_mae = mean_absolute_error(y_val_true_all, y_val_pre_all)
print("-----------------------------------")
print("Overall Validation MSE:", overall_mse)
print("Overall Validation RMSE:", overall_rmse)
print("Overall Validation MAE:", overall_mae)
print("Overall Validation R-squared:", overall_r2)

-----------------------------------
Current item is  1225
Data types of train_dummy after one-hot encoding:
id             object
item_id        object
dept_id        object
d              object
num_sold        int64
wm_yr_wk        int64
month           int64
year            int64
sell_price    float64
event_type    float64
store_id_1       bool
store_id_2       bool
store_id_3       bool
store_id_4       bool
state_id_1       bool
state_id_2       bool
state_id_3       bool
weekday_1        bool
weekday_2        bool
weekday_3        bool
weekday_4        bool
weekday_5        bool
weekday_6        bool
weekday_7        bool
snap_0           bool
snap_1           bool
dtype: object
Index(['d', 'month', 'sell_price', 'event_type', 'store_id_1', 'store_id_2',
       'store_id_3', 'store_id_4', 'state_id_1', 'state_id_2', 'state_id_3',
       'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5',
       'weekday_6', 'weekday_7', 'snap_0', 'snap_1'],
      dtype='object')
Original y_data range: min=0, max=19
Scaled y_data range: min=0.0, max=1.0
Scaled y_val range: min=0.0, max=0.7894736842105263
x_eval_time shape before reshape: (1653, 15, 20)
x_data.shape[1]: 20
LOOKBACK_ARR.shape[0]: 15
item_eval_data.shape: (1933, 14)
store_x_eval.shape: (211, 20)
store_x_eval.shape: (203, 20)
store_x_eval.shape: (200, 20)
store_x_eval.shape: (192, 20)
store_x_eval.shape: (199, 20)
store_x_eval.shape: (171, 20)
store_x_eval.shape: (174, 20)
store_x_eval.shape: (198, 20)
store_x_eval.shape: (216, 20)
store_x_eval.shape: (169, 20)

Model: "sequential_76"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ lstm_153 (LSTM)                      │ (None, 15, 256)             │         283,648 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_58 (Dropout)                 │ (None, 15, 256)             │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ lstm_154 (LSTM)                      │ (None, 256)                 │         525,312 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_59 (Dropout)                 │ (None, 256)                 │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_75 (Dense)                     │ (None, 1)                   │             257 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 809,217 (3.09 MB)

 Trainable params: 809,217 (3.09 MB)

 Non-trainable params: 0 (0.00 B)

Epoch 1/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 7s 53ms/step - loss: 18924.4102 - val_loss: 1972.5114
Epoch 2/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 3934.3872 - val_loss: 43.5837
Epoch 3/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 201.9400 - val_loss: 16.4313
Epoch 4/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 107.3859 - val_loss: 15.2957
Epoch 5/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 81.3130 - val_loss: 8.4849
Epoch 6/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 58.3467 - val_loss: 5.5222
Epoch 7/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 34.1280 - val_loss: 3.1466
Epoch 8/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 22.7189 - val_loss: 1.6834
Epoch 9/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 18.6499 - val_loss: 1.6412
Epoch 10/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 14.3320 - val_loss: 1.2821
Epoch 11/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 52ms/step - loss: 11.1413 - val_loss: 0.9112
Epoch 12/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 8.2863 - val_loss: 1.4349
Epoch 13/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 5.9132 - val_loss: 0.9281
Epoch 14/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 4.5225 - val_loss: 0.1967
Epoch 15/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 3.8019 - val_loss: 0.1853
Epoch 16/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 6s 52ms/step - loss: 3.4947 - val_loss: 0.6657
Epoch 17/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 2.3398 - val_loss: 0.1178
Epoch 18/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 52ms/step - loss: 2.0841 - val_loss: 0.1700
Epoch 19/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.9809 - val_loss: 0.2252
Epoch 20/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.6044 - val_loss: 0.4580
Epoch 21/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.0813 - val_loss: 3.5423
Epoch 22/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.5756 - val_loss: 0.0879
Epoch 23/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.2094 - val_loss: 0.0547
Epoch 24/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.7845 - val_loss: 0.8424
Epoch 25/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 52ms/step - loss: 1.0836 - val_loss: 0.8386
Epoch 26/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.0909 - val_loss: 0.3903
Epoch 27/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.5830 - val_loss: 0.0453
Epoch 28/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.6938 - val_loss: 0.4115
Epoch 29/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.6884 - val_loss: 1.0772
Epoch 30/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.6898 - val_loss: 1.6949
Epoch 31/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.9349 - val_loss: 0.0234
Epoch 32/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.4164 - val_loss: 0.2498
Epoch 33/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.5154 - val_loss: 0.5085
Epoch 34/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.9582 - val_loss: 0.0206
Epoch 35/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.3980 - val_loss: 0.4075
Epoch 36/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 50ms/step - loss: 0.4681 - val_loss: 0.1553
Epoch 37/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.5845 - val_loss: 0.0316
Epoch 38/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.2374 - val_loss: 7.0152
Epoch 39/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 1.8323 - val_loss: 0.0333
Epoch 40/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.1305 - val_loss: 0.7829
Epoch 41/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.5019 - val_loss: 0.0767
Epoch 42/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.6843 - val_loss: 0.1282
Epoch 43/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.7374 - val_loss: 0.0664
Epoch 44/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.2720 - val_loss: 0.0113
Epoch 45/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.2611 - val_loss: 0.0128
Epoch 46/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.3468 - val_loss: 0.6177
Epoch 47/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.1939 - val_loss: 0.1886
Epoch 48/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.3647 - val_loss: 0.1296
Epoch 49/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.5314 - val_loss: 0.5724
Epoch 50/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.2617 - val_loss: 0.7809
Epoch 51/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.8595 - val_loss: 0.2279
Epoch 52/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.3186 - val_loss: 0.0122
Epoch 53/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.1582 - val_loss: 0.0532
Epoch 54/100
106/106 ━━━━━━━━━━━━━━━━━━━━ 5s 51ms/step - loss: 0.1202 - val_loss: 0.8902
53/53 ━━━━━━━━━━━━━━━━━━━━ 1s 12ms/step 
Sample raw predictions (after inverse transform and clipping): [2.2722342 2.019139  2.2932293 0.        0.7174268]
RMSE =  2.5319178
Validation R-squared for item 1225: -0.0014235973358154297
52/52 ━━━━━━━━━━━━━━━━━━━━ 1s 11ms/step
Predicted y_val range (after inverse transform and clipping): min=0.0, max=4.185342311859131
True y_val range (after inverse transform): min=0.0, max=15.000000953674316

-----------------------------------
Current item is  1523
Data types of train_dummy after one-hot encoding:
id             object
item_id        object
dept_id        object
d              object
num_sold        int64
wm_yr_wk        int64
month           int64
year            int64
sell_price    float64
event_type    float64
store_id_1       bool
store_id_2       bool
store_id_3       bool
store_id_4       bool
state_id_1       bool
state_id_2       bool
state_id_3       bool
weekday_1        bool
weekday_2        bool
weekday_3        bool
weekday_4        bool
weekday_5        bool
weekday_6        bool
weekday_7        bool
snap_0           bool
snap_1           bool
dtype: object
Index(['d', 'month', 'sell_price', 'event_type', 'store_id_1', 'store_id_2',
       'store_id_3', 'store_id_4', 'state_id_1', 'state_id_2', 'state_id_3',
       'weekday_1', 'weekday_2', 'weekday_3', 'weekday_4', 'weekday_5',
       'weekday_6', 'weekday_7', 'snap_0', 'snap_1'],
      dtype='object')
Original y_data range: min=0, max=5
Scaled y_data range: min=0.0, max=1.0
Scaled y_val range: min=0.0, max=1.2000000000000002
x_eval_time shape before reshape: (1559, 15, 20)
x_data.shape[1]: 20
LOOKBACK_ARR.shape[0]: 15
item_eval_data.shape: (1839, 14)
store_x_eval.shape: (195, 20)
store_x_eval.shape: (181, 20)
store_x_eval.shape: (181, 20)
store_x_eval.shape: (187, 20)
store_x_eval.shape: (167, 20)
store_x_eval.shape: (180, 20)
store_x_eval.shape: (200, 20)
store_x_eval.shape: (210, 20)
store_x_eval.shape: (160, 20)
store_x_eval.shape: (178, 20)

Model: "sequential_77"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ lstm_155 (LSTM)                      │ (None, 15, 256)             │         283,648 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_60 (Dropout)                 │ (None, 15, 256)             │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ lstm_156 (LSTM)                      │ (None, 256)                 │         525,312 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_61 (Dropout)                 │ (None, 256)                 │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_76 (Dense)                     │ (None, 1)                   │             257 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

y_val_pre_all = np.array(y_val_pre_all, dtype=object)
y_eval_pre_all = np.array(y_eval_pre_all, dtype=object)

def truncate_to_divisible(arr, divisor=28):
    if isinstance(arr, (list, np.ndarray)):
        current_size = len(arr)
        new_size = current_size - (current_size % divisor)
        return arr[:new_size]
    else:
        print(f"Warning: Skipping scalar element: {arr}")
        return np.array([]) # Return an empty array or handle as needed

y_val_output = [truncate_to_divisible(x).reshape(-1, 28) for x in y_val_pre_all if isinstance(x, (list, np.ndarray))]
y_eval_output = [truncate_to_divisible(x).reshape(-1, 28) for x in y_eval_pre_all if isinstance(x, (list, np.ndarray))]

# Check shapes
for i, arr in enumerate(y_val_output[:5]):
    print(f"Array {i}: shape {arr.shape}")

val_id_all = np.array(val_id_all)
eval_id_all = np.array(eval_id_all)

val_id_output = val_id_all.reshape(val_id_all.shape[0]*val_id_all.shape[1])
eval_id_output = eval_id_all.reshape(eval_id_all.shape[0]*eval_id_all.shape[1])
print(val_id_output.shape, eval_id_output.shape)

(1000,) (1000,)

y_val_output = [arr.flatten() for arr in y_val_output]

max_length = max(len(arr) for arr in y_val_output)
padded = [np.pad(arr, (0, max_length - len(arr)), 'constant') for arr in y_val_output]

y_val_output = np.vstack(padded)

if y_val_output.shape[1] % 28 != 0:
    padded_length = ((y_val_output.shape[1] + 27) // 28) * 28
    y_val_output = np.pad(y_val_output, ((0, 0), (0, padded_length - y_val_output.shape[1])), 'constant')
y_val_output = y_val_output.reshape(-1, 28)
num_samples = y_val_output.shape[0]

if len(val_id_output) > num_samples:
    val_id_output = val_id_output[:num_samples]
elif len(val_id_output) < num_samples:
    val_id_output = np.concatenate([val_id_output, np.full(num_samples - len(val_id_output), np.nan)])

output_cols = ['id'] + [f'F{i+1}' for i in range(28)]

val_df = pd.DataFrame(
    {'id': val_id_output},
    columns=['id'] + [f'F{i+1}' for i in range(28)]
)
for i in range(28):
    val_df[f'F{i+1}'] = y_val_output[:, i]

print(val_df.head())
print(f"Final shape: {val_df.shape}")

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[43], line 4
      1 y_val_output = [arr.flatten() for arr in y_val_output]
      3 # 2. Pad all arrays to max length
----> 4 max_length = max(len(arr) for arr in y_val_output)
      5 padded = [np.pad(arr, (0, max_length - len(arr)), 'constant') for arr in y_val_output]
      7 # 3. Stack into 2D array

ValueError: max() iterable argument is empty

y_eval_output = [np.array(arr).flatten() for arr in y_eval_output]

max_length = 0
if y_eval_output:
    max_length = max(len(arr) for arr in y_eval_output)

padded = [np.pad(arr, (0, max_length - len(arr)), 'constant') for arr in y_eval_output]

y_eval_output = np.vstack(padded) if padded else np.empty((0, 0))
if y_eval_output.shape[1] < 28:
    y_eval_output = np.pad(y_eval_output, ((0,0),(0,28-y_eval_output.shape[1])), 'constant')
elif y_eval_output.shape[1] > 28:
    y_eval_output = y_eval_output[:, :28]
num_samples = y_eval_output.shape[0]

if len(eval_id_output) != num_samples:
    print(f"Adjusting IDs: had {len(eval_id_output)}, need {num_samples}")
    if len(eval_id_output) > num_samples:
        eval_id_output = eval_id_output[:num_samples]
    else:
        placeholder_ids = [f"eval_{x}" for x in range(len(eval_id_output), num_samples)]
        eval_id_output = np.concatenate([eval_id_output, placeholder_ids])

eval_df = pd.DataFrame({'id': eval_id_output})
for i in range(28):
    eval_df[f'F{i+1}'] = y_eval_output[:, i]

if 'output_cols' in globals():
    eval_df = eval_df[output_cols]

val_df.to_csv("HOUSEHOLD_output_validation.csv", index =False)
eval_df.to_csv("HOUSEHOLD_output_evauation.csv", index =False)

Adjusting IDs: had 100, need 0

	id	item_id	dept_id	store_id	state_id	d	num_sold	wm_yr_wk	weekday	month	year	snap	sell_price
0	HOUSEHOLD_1_225_WI_3_validation	1225	1	3	2	1422	3	11447	6	12	2014	0	1.64
1	HOUSEHOLD_1_523_TX_3_validation	1523	1	3	3	918	0	11328	6	8	2013	1	20.53
2	HOUSEHOLD_1_248_TX_3_validation	1248	1	3	3	1070	1	11349	4	1	2014	0	8.97
3	HOUSEHOLD_1_538_WI_1_validation	1538	1	1	2	490	1	11218	5	6	2012	0	3.48
4	HOUSEHOLD_2_261_TX_1_validation	2261	2	1	3	383	0	11203	3	2	2012	1	6.97